import sys
from pathlib import Path

base_path = str(Path(__file__).resolve().parent.parent)
sys.path.append(base_path)
from langchain_text_splitters import RecursiveCharacterTextSplitter

custom_separator_aplitter = RecursiveCharacterTextSplitter(
    chunk_size=100,
    chunk_overlap=10,
    separators=["\n##", "\n###", "\n\n", "\n", " ", ""],
    keep_separator=True,
)


def get_markdown_text():
    with open(f"{base_path}/data/06变量.md", "r", encoding="utf-8") as file:
        content = file.read()
        return content


markdown_text = get_markdown_text()

markdown_docs = custom_separator_aplitter.create_documents([markdown_text])

for i, doc in enumerate(markdown_docs):
    print(f"Chunk {i+1}:")
    print(doc.page_content)
    print("-" * 50)
